{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "txt = open('F:/资源/CSCMNews/科技/481681.txt',encoding='utf-8').read() #读取文件\n",
    "import re,math,jieba,jieba.analyse as aly\n",
    "from nltk import *\n",
    "#正则清洗多余换行空格\n",
    "txt=re.sub('\\t|\\n|\\s|\\r',' ',txt)\n",
    "txt=re.sub('\\s+',' ',txt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.jieba自带 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Dumping model to file cache C:\\Users\\Taki\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.819 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "投资 0.21413812578660552\n",
      "天使 0.17551062971133027\n",
      "企业 0.12587852845034403\n",
      "海淀 0.0969584691908257\n",
      "个人 0.08634882003919725\n",
      "VC 0.08225757456123853\n",
      "初创期 0.07469831569472477\n",
      "资本 0.06628322486105505\n",
      "方式 0.06607558251426607\n",
      "中关村 0.06524199801041285\n"
     ]
    }
   ],
   "source": [
    "#TF-IDF\n",
    "for keyword, weight in aly.tfidf(txt, topK=10,withWeight=True):\n",
    "    print('%s %s' % (keyword, weight))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "投资 1.0\n",
      "企业 0.6315427079331518\n",
      "天使 0.3595433102488746\n",
      "资本 0.3175684073412418\n",
      "个人 0.2660339947641947\n",
      "机构 0.24586590243408754\n",
      "方式 0.24379928726739708\n",
      "海淀 0.23613847266608978\n",
      "中国 0.21435838363288562\n",
      "创业 0.1868809845343241\n"
     ]
    }
   ],
   "source": [
    "#TextRank\n",
    "for keyword, weight in aly.textrank(txt, topK=10,withWeight=True):\n",
    "    print('%s %s' % (keyword, weight))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.自行实现TF-IDF "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "txt1 = open('F:/资源/CSCMNews/科技/481681.txt',encoding='utf-8').read() #读取文件\n",
    "txt2 = open('F:/资源/CSCMNews/科技/481682.txt',encoding='utf-8').read() #读取文件\n",
    "txt3 = open('F:/资源/CSCMNews/科技/481687.txt',encoding='utf-8').read() #读取文件\n",
    "txt4 = open('F:/资源/CSCMNews/科技/484041.txt',encoding='utf-8').read() #读取文件\n",
    "txt5 = open('F:/资源/CSCMNews/科技/481685.txt',encoding='utf-8').read() #读取文件\n",
    "txt6 = open('F:/资源/CSCMNews/科技/501688.txt',encoding='utf-8').read() #读取文件\n",
    "txt7 = open('F:/资源/CSCMNews/科技/484892.txt',encoding='utf-8').read() #读取文件\n",
    "txt8 = open('F:/资源/CSCMNews/科技/501714.txt',encoding='utf-8').read() #读取文件\n",
    "txt9 = open('F:/资源/CSCMNews/科技/485311.txt',encoding='utf-8').read() #读取文件\n",
    "txt10 = open('F:/资源/CSCMNews/科技/504333.txt',encoding='utf-8').read() #读取文件\n",
    "stop_words = open('./NLPIR_stopwords.txt','r',encoding='utf-8').read().split('\\n')#导入停用词\n",
    "stringlist=[txt1,txt2,txt3,txt4,txt5,txt6,txt7,txt8,txt9,txt10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cut(txt):#分词并去除停用词\n",
    "    word_list = []\n",
    "    if txt != \"\" and txt is not None:\n",
    "        seg_list = jieba.cut(txt, cut_all=False) #默认模式\n",
    "        for word in seg_list:\n",
    "            if word not in stop_words:\n",
    "                word_list.append(word)\n",
    "    return word_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index,txt in enumerate(stringlist): #批量预处理并分词\n",
    "    txt=re.sub('[a-zA-Z0-9\\\\\\'!\"#$%&\\'()*+,-./:;<=>?@，。?★、…【】《》？“”‘\\\\\\'！[\\\\]^_`{|}~]+','',txt)#去除特殊字符\n",
    "    txt=re.sub('\\t|\\n|\\s|\\r',' ',txt)#正则清洗多余换行空格\n",
    "    txt=re.sub('\\s+',' ',txt)\n",
    "    txt=cut(txt) #分词\n",
    "    fdist=FreqDist(txt) #统计词频\n",
    "    stringlist[index]=fdist #保存对象 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def searchword(txtlist,word):#包含该词文档数\n",
    "    num=1 #分母不能为0\n",
    "    for i in txtlist:\n",
    "        if i[word]!=0:\n",
    "            num+=1\n",
    "    if word=='投资':\n",
    "        print(num)\n",
    "    return num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def TF_IDF(txtlist):\n",
    "    res={}#词字典\n",
    "    for fd in txtlist:#遍历文件列表\n",
    "        for word in fd.keys():#遍历fdist对象\n",
    "            if word not in res: \n",
    "                TF=fd[word]/fd.N()#词频次 或自带.freq方法\n",
    "                IDF=math.log(len(txtlist)/searchword(txtlist,word))#逆文档频率 商取对数\n",
    "                tfidf=TF*IDF\n",
    "                res[word]=tfidf\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\n",
      "镜头  0.13315493520138239\n",
      "云霄  0.09070906108726813\n",
      "塔  0.08293399870835944\n",
      "绿动  0.07799038342191676\n",
      "欧盟  0.07037034595974205\n",
      "欧元  0.06583032363975869\n",
      "客户端  0.06362894072413884\n",
      "手机  0.05614318299188722\n",
      "万台  0.049630243995765204\n",
      "飞机  0.04924206173308841\n"
     ]
    }
   ],
   "source": [
    "data=TF_IDF(stringlist)\n",
    "TFIDF=sorted(data.items(),key=lambda res:res[1],reverse=True)#根据TF-IDF值排序\n",
    "for key, value in TFIDF[:10]:#显示前十\n",
    "    print('%s  %s' % (key, value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.006003790853815741\n"
     ]
    }
   ],
   "source": [
    "print(data['投资'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#找文档的关联词是投资 当所有样本里都含有其时，IDF就为负数"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
